In [30]:
#%% testing loop
print('Start testing...')
model.eval()
losss_test = []
accs_test = []
weights_test = []
with torch.no_grad():
for step_test, (x_test, y_test, mask_test, idx_test) in enumerate(dataloader_test):
print('Test step %i'%(step_test))
#%%%% clean cache of GPU
torch.cuda.empty_cache()
#%%%% forward pass
if type(x_test)==list:
out_test = model.forward([item_.to(config["device"]) for item_ in x_test])
else:
out_test = model.forward(x_test.to(config["device"]))
#%%%% compute loss
loss_test = loss_function(out_test.softmax(1),y_test.squeeze(1).to(config["device"]))
loss_test = (loss_test*mask_test.long().squeeze(1).to(config["device"])).sum() / (torch.count_nonzero(mask_test.long().to(config["device"])))
#%%%% compute metric
if type(metric)==list:
test_acc = [metric_(out_test.cpu().detach(),y_test.cpu().detach(),mask_test.cpu().detach()) for metric_ in metric]
else:
test_acc = metric(out_test.cpu().detach(),y_test.cpu().detach(),mask_test.cpu().detach())
#%%%% printing stuff
print(
"[{}] Test Step: {:d}/{:d}, \tbatch_size: {} \tLoss: {:.4f} \tAcc: {}".format(
dt.datetime.now().strftime("%Y-%m-%dT%H-%M-%S"),
step_test+1,
len(dataloader_test),
dataloader_test.batch_size,
loss_test.mean(),
{metric_.__name__:test_acc_ for metric_,test_acc_ in zip(metric,test_acc)} if type(metric)==list else test_acc
)
)
#%%%% collect loss and accuracy
losss_test.append(loss_test.cpu().detach().numpy())
accs_test.append(test_acc)
weights_test.append(torch.count_nonzero(mask_test).cpu().detach().numpy())
#%%%% plot
#%%%%% calculations for plot
prediction_test = torch.argmax(out_test,1).cpu()
eopatches = [EOPatch.load(dataset_test.paths[idx_.cpu()]) for idx_ in idx_test[0]]
imgs_swir = [eopatch[(FeatureType.DATA,"data")][...,[-1,-3,-4]].squeeze() for eopatch in eopatches]
imgs_true = [eopatch[(FeatureType.DATA,"data")][...,[0,1,2]].squeeze() for eopatch in eopatches]
#%%%%% batch plot
fig, axis = plt.subplots(nrows=4, ncols=dataloader_test.batch_size, figsize=(5*dataloader_test.batch_size,5*4))
axis[0][0].set_ylabel("Prediction")
axis[1][0].set_ylabel("Reference")
axis[2][0].set_ylabel("SWIR Image")
axis[3][0].set_ylabel("True Color Image")
for i in range(dataloader_test.batch_size):
axis[0][i].imshow(prediction_test[i],vmin=0,vmax=config["num_classes"],cmap=config["cmap_reference"])
axis[0][i].set_yticks([])
axis[0][i].set_xticks([])
axis[1][i].imshow(y_test.squeeze(1)[i].cpu(),vmin=0,vmax=config["num_classes"],cmap=config["cmap_reference"])
axis[1][i].set_yticks([])
axis[1][i].set_xticks([])
axis[2][i].imshow(imgs_swir[i]*2.5)
axis[2][i].set_yticks([])
axis[2][i].set_xticks([])
axis[3][i].imshow(imgs_true[i]*2.5)
axis[3][i].set_yticks([])
axis[3][i].set_xticks([])
plt.subplots_adjust(left=0, bottom=0.05, right=1, top=0.95, wspace=0.1, hspace=0)
plt.show()
#%%%% total loss and accuracy
total = np.sum([np.sum(weight_) for weight_ in weights_test])
loss_test_total = np.sum([weight_/total*loss_ for weight_,loss_ in zip(weights_test,losss_test)])
if type(metric)==list:
acc_test_total = [np.sum([weight_/total*acc_[i] for weight_,acc_ in zip(weights_test,accs_test)]) for i in range(len(metric))]
else:
acc_test_total = np.sum([weight_/total*acc_ for weight_,acc_ in zip(weights_test,accs_test)])
# print total values
print(
"[{}] Test: \tTotal Loss: {:.4f} \tTotal Acc: {}".format(
dt.datetime.now().strftime("%Y-%m-%dT%H-%M-%S"),
loss_test_total,
{metric_.__name__:test_acc_ for metric_,test_acc_ in zip(metric,acc_test_total)} if type(metric)==list else acc_test_total
)
)
#%%% write to tensorboard
#%%%% log loss
writer.add_scalar(f'LossTest/{type(loss_function).__name__}', loss_test_total, global_step=step_test)
#%%%% log metric
if type(metric)==list:
writer.add_scalars('AccuracyTest',{metric_.__name__:test_acc_ for metric_,test_acc_ in zip(metric,acc_test_total)},global_step=step_test)
else:
writer.add_scalar('AccuracyTest', acc_test_total, global_step=step_test)
print()
Start testing...
Test step 0
[2023-02-13T15-44-07] Test Step: 1/2, batch_size: 4 Loss: 0.9858 Acc: {'accuracy': tensor(0.7514), 'cohen_kappa': 0.5258390902529031}
Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers). Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers). Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers). Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers). Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers). Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers). Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers). Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).
Test step 1
[2023-02-13T15-44-09] Test Step: 2/2, batch_size: 4 Loss: 0.9805 Acc: {'accuracy': tensor(0.7578), 'cohen_kappa': 0.6144480004861541}
Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers). Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers). Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers). Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers). Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers). Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers). Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers). Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).
[2023-02-13T15-44-11] Test: Total Loss: 0.9832 Total Acc: {'accuracy': 0.754561, 'cohen_kappa': 0.5697383328759842}